﻿using System;
using System.Collections.Specialized;
using System.IO;
using System.Net;
using System.Threading.Tasks;

namespace OMR.Crawler.Core.Crawlers
{
    public class WebCrawler : CrawlerBase
    {
        private readonly string _rootUrl;
        private StringCollection _visitedUrls;
        private CookieContainer _cookieContainer;
        private string[] _allowedMimeTypes;

        public WebCrawler(MyCollection settings)
            : base(settings)
        {
            _rootUrl = (string)settings.GetItem("EntryPoint").Value;
            _visitedUrls = new StringCollection();
            _cookieContainer = new CookieContainer();
            _allowedMimeTypes = new string[] { "text/html", "text/plain" };

            CrawlQueue.Enqueue(new CrawlWork() { EntryPoint = _rootUrl, DeepCount = 0 });
        }

        public override void Begin()
        {
            base.Begin((currentWork) => Iterate(currentWork));
        }

        private void Iterate(CrawlWork currentWork)
        {
            //if (currentWork == null)
            //    return;

            if (!LoadContent(ref currentWork))
            {
                
                return;
            }

            ItemCrawled(currentWork);

            if (currentWork.MimeType == null)
                return;

            if (currentWork.MimeType == "text/html")
            {
                DeepLink(currentWork);
            }
            else
            {
                return;
            }
        }

        private void DeepLink(CrawlWork currentWork)
        {
            if (currentWork.Content == null)
                return;

            var htmlDocument = new HtmlAgilityPack.HtmlDocument();

            using (var ms = new MemoryStream(currentWork.Content))
            {
                try
                {
                    htmlDocument.Load(ms);
                }
                catch (Exception exception)
                {
                    CrawlError(exception, currentWork.EntryPoint);
                    return;
                }
            }// using

            //TODO: parallel
            var collection = htmlDocument.DocumentNode.SelectNodes("//a[@href]");

            if (collection == null)
                return;

            //foreach (var item in collection.ToArray())
            Parallel.ForEach(collection, (item) =>
            {
                try
                {
                    var absoluteUrl = new Uri(new Uri(currentWork.EntryPoint), item.Attributes["href"].Value);

                    // is extarnal site link?
                    if (!new Uri(_rootUrl).IsBaseOf(absoluteUrl))
                        return; //continue;

                    // is anchored link?
                    int anchorIndex = absoluteUrl.ToString().IndexOf("#");
                    if (anchorIndex > -1)
                        absoluteUrl = new Uri(absoluteUrl.ToString().Substring(0, anchorIndex));

                    if (_visitedUrls.Contains(absoluteUrl.ToString()))
                        return; //continue;

                    _visitedUrls.Add(absoluteUrl.ToString());

                    var subCrawlWork = new CrawlWork();
                    subCrawlWork.DeepCount = currentWork.DeepCount + 1;
                    subCrawlWork.EntryPoint = absoluteUrl.ToString();
                    subCrawlWork.Content = new byte[] { };

                    //TODO: içerik tipini kontrol et
                    System.Diagnostics.Trace.WriteLine(absoluteUrl);
                    CrawlQueue.Enqueue(subCrawlWork);
                }
                catch (Exception exception)
                {
                    CrawlError(exception, currentWork.EntryPoint);
                }
            });
        }

        // TODO: async
        bool LoadContent(ref CrawlWork currentWork)
        {
            try
            {
                var response = GetResponse(currentWork.EntryPoint); // TODO: async?

                if (response.ContentType == null)
                {
                    response.Close();
                    return false;
                }

                currentWork.MimeType = response.ContentType.Split(';')[0].Trim();

                if (!IsDownloadSupported(currentWork.MimeType))
                {
                    response.Close();
                    return false;
                }

                var buffer = new byte[2048];

                using (var stream = response.GetResponseStream())
                {
                    using (var ms = new MemoryStream())
                    {
                        int read;
                        while ((read = stream.Read(buffer, 0, buffer.Length)) > 0)
                        {
                            ms.Write(buffer, 0, read);
                        }

                        currentWork.Content = ms.ToArray();
                    }
                }

                return true;
            }
            catch (Exception exception)
            {
                CrawlError(exception, currentWork.EntryPoint);
                return false;
            }
        }

        //TODO: allow with mimetype white/black list
        private bool IsDownloadSupported(string mimeType)
        {
            if (mimeType.StartsWith("image"))
            {
                return false;
            }

            return true;
        }

        private WebResponse GetResponse(string url)
        {
            var request = (HttpWebRequest)WebRequest.Create(url);
            request.AllowAutoRedirect = true; //TODO: setting
            request.CookieContainer = _cookieContainer;
            request.KeepAlive = false; //TODO: setting
            request.Method = "GET"; //TODO: setting
            //request.Proxy  //TODO:
            //request.UseDefaultCredentials = true; //TODO: setting
            //request.Credentials //TODO: 
            request.Timeout = 30000; //TODO: setting
            request.UserAgent = "tst";

            return request.GetResponse();
        }

    }
}